{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据来源于Kaggle竞赛：Event Recommendation Engine Challenge，根据\n",
    "    events they’ve responded to in the past\n",
    "    user demographic information\n",
    "    what events they’ve seen and clicked on in our app\n",
    "预测用户对某个活动是否感兴趣\n",
    "\n",
    "竞赛官网：\n",
    "https://www.kaggle.com/c/event-recommendation-engine-challenge/data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#数据量太大，pandas不能一次将所有数据读入\n",
    "#也可以用pandas,一次读取部分数据，可以参考：https://www.cnblogs.com/datablog/p/6127000.html\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "import scipy.sparse as ss\n",
    "import scipy.io as sio\n",
    "\n",
    "#保存数据\n",
    "import pickle\n",
    "\n",
    "#event的特征需要编码\n",
    "from utils import FeatureEng\n",
    "from sklearn.preprocessing import normalize\n",
    "#相似度/距离\n",
    "import scipy.spatial.distance as ssd\n",
    "from __future__ import division\n",
    "\n",
    "from numpy.random import random  \n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 统计活动数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of records :3137972\n"
     ]
    }
   ],
   "source": [
    "#读取数据，并统计有多少不同的events\n",
    "#其实EDA.ipynb中用read_csv已经统计过了\n",
    "lines = 0\n",
    "fin = open(\"events.csv\", 'r')\n",
    "#找到用C/C++的感觉了\n",
    "#字段：event_id, user_id,start_time, city, state, zip, country, lat, and lng， 101 columns of words count\n",
    "fin.readline() # skip header，列名行\n",
    "for line in fin:\n",
    "    cols = line.strip().split(\",\")\n",
    "    lines += 1\n",
    "fin.close()\n",
    "\n",
    "print(\"number of records :%d\" % lines)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读取之前算好的测试集和训练集中出现过的活动"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的活动列表\n",
    "eventIndex = pickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "n_events = len(eventIndex)\n",
    "\n",
    "print(\"number of events in train & test :%d\" % n_events)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 处理events.csv --> 特征编码、活动之间的相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "FE = FeatureEng()\n",
    "\n",
    "fin = open(\"events.csv\", 'r')\n",
    "\n",
    "#字段：event_id, user_id,start_time, city, state, zip, country, lat, and lng， 101 columns of words count\n",
    "fin.readline() # skip header\n",
    "\n",
    "#start_time, city, state, zip, country, lat, and lng\n",
    "eventPropMatrix = ss.dok_matrix((n_events, 7))\n",
    "\n",
    "#词频特征\n",
    "eventContMatrix = ss.dok_matrix((n_events, 101))\n",
    "\n",
    "for line in fin.readlines():       #fin.readlines()返回一个列表，这是可以list(fin)或者直接迭代文件\n",
    "    cols = line.strip().split(\",\")\n",
    "    eventId = str(cols[0])     #这里转换成字符串不是很明白\n",
    "    \n",
    "    #if eventIndex.has_key(eventId):  #在训练集或测试集中出现\n",
    "    if eventId in eventIndex:\n",
    "        i = eventIndex[eventId]\n",
    "  \n",
    "        #event的特征编码，这里只是简单处理，其实开始时间，地点等信息很重要\n",
    "        eventPropMatrix[i, 0] = FE.getJoinedYearMonth(cols[2]) # start_time\n",
    "        eventPropMatrix[i, 1] = FE.getFeatureHash(cols[3]) # city\n",
    "        eventPropMatrix[i, 2] = FE.getFeatureHash(cols[4]) # state\n",
    "        eventPropMatrix[i, 3] = FE.getFeatureHash(cols[5]) # zip\n",
    "        eventPropMatrix[i, 4] = FE.getFeatureHash(cols[6]) # country\n",
    "        eventPropMatrix[i, 5] = FE.getFloatValue(cols[7]) # lat\n",
    "        eventPropMatrix[i, 6] = FE.getFloatValue(cols[8]) # lon\n",
    "        \n",
    "        #词频\n",
    "        for j in range(9, 110):\n",
    "            eventContMatrix[i, j-9] = cols[j]\n",
    "fin.close()\n",
    " \n",
    " \n",
    "#用L2模归一化,Kmeans聚类基于L2距离\n",
    "eventPropMatrix = normalize(eventPropMatrix,\n",
    "    norm=\"l2\", axis=0, copy=False)\n",
    "sio.mmwrite(\"EV_eventPropMatrix\", eventPropMatrix)\n",
    "\n",
    "#词频，可以考虑我们用这部分特征进行聚类，得到活动的genre（法语词汇genre ：种类、类型）\n",
    "eventContMatrix = normalize(eventContMatrix,\n",
    "    norm=\"l2\", axis=0, copy=False)\n",
    "sio.mmwrite(\"EV_eventContMatrix\", eventContMatrix)\n",
    "\n",
    "\n",
    "# calculate similarity between event pairs based on the two matrices\n",
    "eventPropSim = ss.dok_matrix((n_events, n_events))\n",
    "eventContSim = ss.dok_matrix((n_events, n_events))\n",
    "\n",
    "#读取在测试集和训练集中出现的活动对\n",
    "uniqueEventPairs = pickle.load(open(\"PE_uniqueEventPairs.pkl\", 'rb'))\n",
    "\n",
    "for e1, e2 in uniqueEventPairs:\n",
    "    #i = eventIndex[e1]\n",
    "    #j = eventIndex[e2]\n",
    "    i = e1\n",
    "    j = e2\n",
    "    \n",
    "    #非词频特征，采用Person相关系数作为相似度\n",
    "    #if not eventPropSim.has_key((i,j)):  python2写法\n",
    "    if (i,j) not in eventPropSim:                #还不明白这个判断语句的意思\n",
    "        epsim = ssd.correlation(eventPropMatrix.getrow(i).todense(),\n",
    "            eventPropMatrix.getrow(j).todense())\n",
    "        \n",
    "        eventPropSim[i, j] = epsim\n",
    "        eventPropSim[j, i] = epsim\n",
    "    \n",
    "    #对词频特征，采用余弦相似度，也可以用直方图交/Jacard相似度\n",
    "    #if not eventContSim.has_key((i,j)):\n",
    "    if (i,j) not in eventContSim:\n",
    "        ecsim = ssd.cosine(eventContMatrix.getrow(i).todense(),\n",
    "            eventContMatrix.getrow(j).todense())\n",
    "    \n",
    "        eventContSim[i, j] = epsim\n",
    "        eventContSim[j, i] = epsim\n",
    "    \n",
    "sio.mmwrite(\"EV_eventPropSim\", eventPropSim)\n",
    "sio.mmwrite(\"EV_eventContSim\", eventContSim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eventPropSim.getrow(0).todense()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用户数据处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of users in train & test :3391\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的用户列表\n",
    "userIndex = pickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "n_users = len(userIndex)\n",
    "\n",
    "print(\"number of users in train & test :%d\" % n_users)\n",
    "\n",
    "#读取数据\n",
    "users = pd.read_csv(\"users.csv\")\n",
    "users.head()\n",
    "\n",
    "FE = FeatureEng()\n",
    "\n",
    "#locale\tbirthyear\tgender\tjoinedAt\tlocation\ttimezone\n",
    "#去掉user_id列\n",
    "n_cols = users.shape[1] - 1\n",
    "cols = ['LocaleId', 'BirthYearInt', 'GenderId', 'JoinedYearMonth', 'CountryId', 'TimezoneInt']\n",
    "\n",
    "#users编码后的特征\n",
    "#userMatrix = np.zeros((n_users, n_cols), dtype=np.int)\n",
    "userMatrix = ss.dok_matrix((n_users, n_cols))\n",
    "\n",
    "for u in range(users.shape[0]):      #users.shape[0]：users行数\n",
    "    userId = str(users.loc[u,'user_id'])       #loc的下标对象是一个元祖，分别于dataframe的两个轴对应\n",
    "    \n",
    "    #if userIndex.has_key(userId):       python2写法\n",
    "    if userId in userIndex:      #在训练集或测试集中出现\n",
    "        i = userIndex[userId]\n",
    "    \n",
    "        userMatrix[i, 0] = FE.getLocaleId(users.loc[u,'locale'])\n",
    "        userMatrix[i, 1] = FE.getBirthYearInt(users.loc[u,'birthyear'])\n",
    "        userMatrix[i, 2] = FE.getGenderId(users.loc[u,'gender'])\n",
    "        userMatrix[i, 3] = FE.getJoinedYearMonth(users.loc[u,'joinedAt'])\n",
    "        \n",
    "        #由于地点的写法不规范，该编码似乎不起作用（所有样本的特征都被编码成0了）\n",
    "        userMatrix[i, 4] = FE.getCountryId(users.loc[u,'location'])\n",
    "        \n",
    "        userMatrix[i, 5] = FE.getTimezoneInt(users.loc[u,'timezone'])\n",
    "\n",
    "# 归一化用户矩阵\n",
    "userMatrix = normalize(userMatrix, norm=\"l2\", axis=0, copy=False)\n",
    "sio.mmwrite(\"US_userMatrix\", userMatrix)\n",
    "\n",
    "\n",
    "# 计算用户相似度矩阵，之后用户推荐系统\n",
    "userSimMatrix = ss.dok_matrix((n_users, n_users))\n",
    "\n",
    "#读取在测试集和训练集中出现的用户对\n",
    "uniqueUserPairs = pickle.load(open(\"FE_uniqueUserPairs.pkl\", 'rb'))\n",
    "\n",
    "#对角线元素\n",
    "for i in range(0, n_users):\n",
    "    userSimMatrix[i, i] = 1.0\n",
    "    \n",
    "#对称\n",
    "for u1, u2 in uniqueUserPairs:\n",
    "    #i = userIndex[u1]\n",
    "    #j = userIndex[u2]\n",
    "    i = u1\n",
    "    j = u2\n",
    "    if (i,j) not in userSimMatrix:\n",
    "        #Person相关系数做为相似度度量\n",
    "        #特征：国家（locale、location）、年龄、性别、时区、地点\n",
    "        #usim = ssd.correlation(userMatrix[i,:],\n",
    "            #userMatrix[j,:])\n",
    "    \n",
    "        usim = ssd.correlation(userMatrix.getrow(i).todense(),\n",
    "          userMatrix.getrow(j).todense())\n",
    "        userSimMatrix[i, j] = usim\n",
    "        userSimMatrix[j, i] = usim\n",
    "    \n",
    "sio.mmwrite(\"US_userSimMatrix\", userSimMatrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 用户社交数据（user_friends.csv）处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of users in train & test :3391\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的事件列表\n",
    "userIndex = pickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "n_users = len(userIndex)\n",
    "\n",
    "print(\"number of users in train & test :%d\" % n_users)\n",
    "\n",
    "#用户-事件关系矩阵\n",
    "userEventScores = sio.mmread(\"PE_userEventScores\")\n",
    "\n",
    "#后续用于将用户朋友参加的活动影响到用户\n",
    "eventsForUser = pickle.load(open(\"PE_eventsForUser.pkl\", 'rb'))\n",
    "\n",
    "#读取数据\n",
    "\n",
    "\"\"\"\n",
    "  找出某用户的那些朋友\n",
    "  1)如果你有更多的朋友，可能你性格外向，更容易参加各种活动\n",
    "  2)如果你朋友会参加某个活动，可能你也会跟随去参加一下\n",
    "\"\"\"\n",
    " \n",
    "#用户有多少个朋友\n",
    "numFriends = np.zeros((n_users))       #生成n_users个元素的零矩阵\n",
    "userFriends = ss.dok_matrix((n_users, n_users))\n",
    "    \n",
    "fin = open(\"user_friends.csv\", 'r')\n",
    "#字段：user，friends\n",
    "fin.readline()                # skip header\n",
    "\n",
    "#ln = 0\n",
    "for line in fin:  #对每个用户        \n",
    "    cols = line.strip().split(\",\")\n",
    "    user = str(cols[0])    #user\n",
    "    \n",
    "    if user in userIndex:   #该用户在训练集和测试集的用户列表中\n",
    "        friends = cols[1].split(\" \")  #friends\n",
    "        i = userIndex[user]       #该用户的索引\n",
    "        numFriends[i] = len(friends)\n",
    "        for friend in friends:  #该用户的每个朋友\n",
    "            str_friend = str(friend)\n",
    "            if str_friend in userIndex:  #如果朋友也在训练集或测试集中出现\n",
    "                j = userIndex[str_friend]   #朋友的索引\n",
    "            \n",
    "                # the objective of this score is to infer the degree to\n",
    "                # and direction in which this friend will influence the\n",
    "                # user's decision, so we sum the user/event score for\n",
    "                # this user across all training events.\n",
    "            \n",
    "                #userEventScores为用户对活动的打分（interested - not interseted）\n",
    "                #在Users-Events.ipynb中计算好了\n",
    "                eventsForUser = userEventScores.getrow(j).todense()\n",
    "            \n",
    "                #所有朋友参加活动的数量（平均频率）\n",
    "                score = eventsForUser.sum() / np.shape(eventsForUser)[1]\n",
    "                userFriends[i, j] += score\n",
    "                userFriends[i, j] += score\n",
    "            \n",
    "fin.close()\n",
    "    \n",
    "\n",
    "#用户的朋友数目\n",
    "# 归一化数组\n",
    "sumNumFriends = numFriends.sum(axis=0)\n",
    "numFriends = numFriends / sumNumFriends\n",
    "sio.mmwrite(\"UF_numFriends\", np.matrix(numFriends))\n",
    "\n",
    "#\n",
    "userFriends = normalize(userFriends, norm=\"l2\", axis=0, copy=False)\n",
    "sio.mmwrite(\"UF_userFriends\", userFriends)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 活动热度数据（event_attendees.csv）处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的事件列表\n",
    "eventIndex = pickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "n_events = len(eventIndex)\n",
    "\n",
    "print(\"number of events in train & test :%d\" % n_events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取数据\n",
    "\"\"\"\n",
    "  统计某个活动，参加和不参加的人数，计算活动热度\n",
    "\"\"\"\n",
    "\n",
    "#活动活跃度\n",
    "eventPopularity = ss.dok_matrix((n_events, 1))\n",
    "    \n",
    "f = open(\"event_attendees.csv\", 'r')\n",
    "\n",
    "#字段：event_id,yes, maybe, invited, and no\n",
    "f.readline() # skip header\n",
    "\n",
    "for line in f:\n",
    "    cols = line.strip().split(\",\")\n",
    "    eventId = str(cols[0])   #event_id\n",
    "    if eventId in eventIndex:\n",
    "        i = eventIndex[eventId]  #事件索引\n",
    "        \n",
    "        #yes - no\n",
    "        eventPopularity[i, 0] = \\\n",
    "          len(cols[1].split(\" \")) - len(cols[4].split(\" \"))\n",
    "    \n",
    "f.close()\n",
    "    \n",
    "eventPopularity = normalize(eventPopularity, norm=\"l1\",\n",
    "      axis=0, copy=False)\n",
    "sio.mmwrite(\"EA_eventPopularity\", eventPopularity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 将所有特征串联起来\n",
    "构成RS_Train.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将所有特征串联起来，构成RS_Train.csv\n",
    "#RS_Test.csv\n",
    "#为最后推荐系统做准备\n",
    "from __future__ import division\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "from numpy.random import random  \n",
    "from collections import defaultdict\n",
    "\n",
    "class RecommonderSystem:\n",
    "  def __init__(self):\n",
    "    # 读入数据做初始化\n",
    "    \n",
    "    #用户和活动新的索引\n",
    "    self.userIndex = pickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "    self.eventIndex = pickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "    self.n_users = len(self.userIndex)\n",
    "    self.n_items = len(self.eventIndex)\n",
    "    \n",
    "    #用户-活动关系矩阵R\n",
    "    #在train_SVD会重新从文件中读取,二者要求的格式不同，来不及统一了:(\n",
    "    self.userEventScores = sio.mmread(\"PE_userEventScores\").todense()\n",
    "    \n",
    "    #倒排表\n",
    "    ##每个用户参加的事件\n",
    "    self.itemsForUser = pickle.load(open(\"PE_eventsForUser.pkl\", 'rb'))\n",
    "    ##事件参加的用户\n",
    "    self.usersForItem = pickle.load(open(\"PE_usersForEvent.pkl\", 'rb'))\n",
    "    \n",
    "    #基于模型的协同过滤参数初始化,训练\n",
    "    self.init_SVD()\n",
    "    self.train_SVD(trainfile = \"train.csv\")\n",
    "    \n",
    "    #根据用户属性计算出的用户之间的相似度\n",
    "    self.userSimMatrix = sio.mmread(\"US_userSimMatrix\").todense()\n",
    "    \n",
    "    #根据活动属性计算出的活动之间的相似度\n",
    "    self.eventPropSim = sio.mmread(\"EV_eventPropSim\").todense()\n",
    "    self.eventContSim = sio.mmread(\"EV_eventContSim\").todense()\n",
    "    \n",
    "    #每个用户的朋友的数目\n",
    "    self.numFriends = sio.mmread(\"UF_numFriends\")\n",
    "    #用户的每个朋友参加活动的分数对该用户的影响\n",
    "    self.userFriends = sio.mmread(\"UF_userFriends\").todense()\n",
    "    \n",
    "    #活动本身的热度\n",
    "    self.eventPopularity = sio.mmread(\"EA_eventPopularity\").todense()\n",
    "\n",
    "  def init_SVD(self, K=20):\n",
    "    #初始化模型参数（for 基于模型的协同过滤SVD_CF）\n",
    "    self.K = K  \n",
    "    \n",
    "    #init parameters\n",
    "    #bias\n",
    "    self.bi = np.zeros(self.n_items)  \n",
    "    self.bu = np.zeros(self.n_users)  \n",
    "    \n",
    "    #the small matrix\n",
    "    self.P = random((self.n_users,self.K))/10*(np.sqrt(self.K))\n",
    "    self.Q = random((self.K, self.n_items))/10*(np.sqrt(self.K))  \n",
    "                  \n",
    "          \n",
    "  def train_SVD(self,trainfile = 'train.csv', steps=100,gamma=0.04,Lambda=0.15):\n",
    "    #训练SVD模型（for 基于模型的协同过滤SVD_CF）\n",
    "    #gamma：为学习率\n",
    "    #Lambda：正则参数\n",
    "    \n",
    "    #偷懒了，为了和原来的代码的输入接口一样，直接从训练文件中去读取数据\n",
    "    print (\"SVD Train...\")\n",
    "    ftrain = open(trainfile, 'r')\n",
    "    ftrain.readline()\n",
    "    self.mu = 0.0\n",
    "    n_records = 0\n",
    "    uids = []  #每条记录的用户索引\n",
    "    i_ids = [] #每条记录的item索引\n",
    "    #用户-Item关系矩阵R（内容同userEventScores相同），临时变量，训练完了R不再需要\n",
    "    R = np.zeros((self.n_users, self.n_items))\n",
    "    \n",
    "    for line in ftrain:\n",
    "        cols = line.strip().split(\",\")\n",
    "        u = self.userIndex[cols[0]]  #用户\n",
    "        i = self.eventIndex[cols[1]] #活动\n",
    "        \n",
    "        uids.append(u)\n",
    "        i_ids.append(i)\n",
    "        \n",
    "        R[u,i] = int(cols[4])  #interested\n",
    "        self.mu += R[u,i]\n",
    "        n_records += 1\n",
    "    \n",
    "    ftrain.close()\n",
    "    self.mu /= n_records\n",
    "    \n",
    "    # 请补充完整SVD模型训练过程\n",
    "    print(\"SVD trained\" )\n",
    "    \n",
    "  def pred_SVD(self, uid, i_id):\n",
    "    #根据当前参数，预测用户uid对Item（i_id）的打分        \n",
    "    ans=self.mu + self.bi[i_id] + self.bu[uid] + np.dot(self.P[uid,:],self.Q[:,i_id])  \n",
    "        \n",
    "    #将打分范围控制在0-1之间\n",
    "    if ans>1:  \n",
    "        return 1  \n",
    "    elif ans<0:  \n",
    "        return 0\n",
    "    return ans  \n",
    "\n",
    "  def sim_cal_UserCF(self, uid1, uid2 ):\n",
    "    #请补充基于用户的协同过滤中的两个用户uid1和uid2之间的相似度（根据两个用户对item打分的相似度）\n",
    "    similarity = 0.0\n",
    "    \n",
    "    \n",
    "    \n",
    "    return similarity  \n",
    "\n",
    "  def userCFReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    根据User-based协同过滤，得到event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i\n",
    "      for every other user v that has a preference for i\n",
    "        compute similarity s between u and v\n",
    "        incorporate v's preference for i weighted by s into running aversge\n",
    "    return top items ranked by weighted average\n",
    "    \"\"\"\n",
    "    #请补充完整代码\n",
    "    ans = 0.0\n",
    "  \n",
    "    return ans\n",
    "\n",
    "\n",
    "  def sim_cal_ItemCF(self, i_id1, i_id2):\n",
    "    #计算Item i_id1和i_id2之间的相似性\n",
    "    #请补充完整代码\n",
    "     similarity = 0.0\n",
    "        \n",
    "     return similarity     \n",
    "        \n",
    "     return num/den  \n",
    "            \n",
    "  def eventCFReco(self, userId, eventId):    \n",
    "    \"\"\"\n",
    "    根据基于物品的协同过滤，得到Event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i \n",
    "        for every item j tht u has a preference for\n",
    "            compute similarity s between i and j\n",
    "            add u's preference for j weighted by s to a running average\n",
    "    return top items, ranked by weighted average\n",
    "    \"\"\"\n",
    "    #请补充完整代码\n",
    "    ans = 0.0\n",
    "  \n",
    "    return ans\n",
    "    \n",
    "  def svdCFReco(self, userId, eventId):\n",
    "    #基于模型的协同过滤, SVD++/LFM\n",
    "    u = self.userIndex[userId]\n",
    "    i = self.eventIndex[eventId]\n",
    "\n",
    "    return self.pred_SVD(u,i)\n",
    "\n",
    "  def userReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    类似基于User-based协同过滤，只是用户之间的相似度由用户本身的属性得到，计算event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i\n",
    "      for every other user v that has a preference for i\n",
    "        compute similarity s between u and v\n",
    "        incorporate v's preference for i weighted by s into running aversge\n",
    "    return top items ranked by weighted average\n",
    "    \"\"\"\n",
    "    i = self.userIndex[userId]\n",
    "    j = self.eventIndex[eventId]\n",
    "\n",
    "    vs = self.userEventScores[:, j]\n",
    "    sims = self.userSimMatrix[i, :]\n",
    "\n",
    "    prod = sims * vs\n",
    "\n",
    "    try:\n",
    "      return prod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      return 0\n",
    "\n",
    "  def eventReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    类似基于Item-based协同过滤，只是item之间的相似度由item本身的属性得到，计算Event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i \n",
    "      for every item j that u has a preference for\n",
    "        compute similarity s between i and j\n",
    "        add u's preference for j weighted by s to a running average\n",
    "    return top items, ranked by weighted average\n",
    "    \"\"\"\n",
    "    i = self.userIndex[userId]\n",
    "    j = self.eventIndex[eventId]\n",
    "    js = self.userEventScores[i, :]\n",
    "    psim = self.eventPropSim[:, j]\n",
    "    csim = self.eventContSim[:, j]\n",
    "    pprod = js * psim\n",
    "    cprod = js * csim\n",
    "    \n",
    "    pscore = 0\n",
    "    cscore = 0\n",
    "    try:\n",
    "      pscore = pprod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      pass\n",
    "    try:\n",
    "      cscore = cprod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      pass\n",
    "    return pscore, cscore\n",
    "\n",
    "  def userPop(self, userId):\n",
    "    \"\"\"\n",
    "    基于用户的朋友个数来推断用户的社交程度\n",
    "    主要的考量是如果用户的朋友非常多，可能会更倾向于参加各种社交活动\n",
    "    \"\"\"\n",
    "    if userId in self.userIndex:\n",
    "      i = self.userIndex[userId]\n",
    "      try:\n",
    "        return self.numFriends[0, i]\n",
    "      except IndexError:\n",
    "        return 0\n",
    "    else:\n",
    "      return 0\n",
    "\n",
    "  def friendInfluence(self, userId):\n",
    "    \"\"\"\n",
    "    朋友对用户的影响\n",
    "    主要考虑用户所有的朋友中，有多少是非常喜欢参加各种社交活动/event的\n",
    "    用户的朋友圈如果都积极参与各种event，可能会对当前用户有一定的影响\n",
    "    \"\"\"\n",
    "    nusers = np.shape(self.userFriends)[1]\n",
    "    i = self.userIndex[userId]\n",
    "    return (self.userFriends[i, :].sum(axis=0) / nusers)[0,0]\n",
    "\n",
    "  def eventPop(self, eventId):\n",
    "    \"\"\"\n",
    "    本活动本身的热度\n",
    "    主要是通过参与的人数来界定的\n",
    "    \"\"\"\n",
    "    i = self.eventIndex[eventId]\n",
    "    return self.eventPopularity[i, 0]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateRSData(RS, train=True, header=True):\n",
    "    \"\"\"\n",
    "    把前面user-based协同过滤 和 item-based协同过滤，以及各种热度和影响度作为特征组合在一起\n",
    "    生成新的训练数据，用于分类器分类使用\n",
    "    \"\"\"\n",
    "    fn = \"train.csv\" if train else \"test.csv\"\n",
    "    fin = open(fn, 'r')\n",
    "    fout = open(\"RS_\" + fn, 'w')\n",
    "    \n",
    "    #忽略第一行（列名字）\n",
    "    fin.readline().strip().split(\",\")\n",
    "    \n",
    "    # write output header\n",
    "    if header:\n",
    "      ocolnames = [\"invited\", \"userCF_reco\", \"evtCF_reco\",\"svdCF_reco\",\"user_reco\", \"evt_p_reco\",\n",
    "        \"evt_c_reco\", \"user_pop\", \"frnd_infl\", \"evt_pop\"]\n",
    "      if train:\n",
    "        ocolnames.append(\"interested\")\n",
    "        ocolnames.append(\"not_interested\")\n",
    "      fout.write(\",\".join(ocolnames) + \"\\n\")\n",
    "    \n",
    "    ln = 0\n",
    "    for line in fin:\n",
    "      ln += 1\n",
    "      if ln%500 == 0:\n",
    "          print (\"%s:%d (userId, eventId)=(%s, %s)\" % (fn, ln, userId, eventId))\n",
    "          #break;\n",
    "      \n",
    "      cols = line.strip().split(\",\")\n",
    "      userId = cols[0]\n",
    "      eventId = cols[1]\n",
    "      invited = cols[2]\n",
    "      \n",
    "      userCF_reco = RS.userCFReco(userId, eventId)\n",
    "      itemCF_reco = RS.eventCFReco(userId, eventId)\n",
    "      svdCF_reco = RS.svdCFReco(userId, eventId)\n",
    "        \n",
    "      user_reco = RS.userReco(userId, eventId)\n",
    "      evt_p_reco, evt_c_reco = RS.eventReco(userId, eventId)\n",
    "      user_pop = RS.userPop(userId)\n",
    "     \n",
    "      frnd_infl = RS.friendInfluence(userId)\n",
    "      evt_pop = RS.eventPop(eventId)\n",
    "      ocols = [invited, userCF_reco, itemCF_reco, svdCF_reco,user_reco, evt_p_reco,\n",
    "        evt_c_reco, user_pop, frnd_infl, evt_pop]\n",
    "      \n",
    "      if train:\n",
    "        ocols.append(cols[4]) # interested\n",
    "        ocols.append(cols[5]) # not_interested\n",
    "      fout.write(\",\".join(map(lambda x: str(x), ocols)) + \"\\n\")\n",
    "    \n",
    "    fin.close()\n",
    "    fout.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVD Train...\n",
      "SVD trained\n",
      "生成训练数据...\n",
      "\n",
      "train.csv:500 (userId, eventId)=(123290209, 1887085024)\n",
      "train.csv:1000 (userId, eventId)=(272886293, 199858305)\n",
      "train.csv:1500 (userId, eventId)=(395305791, 1582270949)\n",
      "train.csv:2000 (userId, eventId)=(527523423, 3272728211)\n",
      "train.csv:2500 (userId, eventId)=(651258472, 792632006)\n",
      "train.csv:3000 (userId, eventId)=(811791433, 524756826)\n",
      "train.csv:3500 (userId, eventId)=(985547042, 1269035551)\n",
      "train.csv:4000 (userId, eventId)=(1107615001, 173949238)\n",
      "train.csv:4500 (userId, eventId)=(1236336671, 3849306291)\n",
      "train.csv:5000 (userId, eventId)=(1414301782, 2652356640)\n",
      "train.csv:5500 (userId, eventId)=(1595465532, 955398943)\n",
      "train.csv:6000 (userId, eventId)=(1747091728, 2131379889)\n",
      "train.csv:6500 (userId, eventId)=(1914182220, 955398943)\n",
      "train.csv:7000 (userId, eventId)=(2071842684, 1076364848)\n",
      "train.csv:7500 (userId, eventId)=(2217853337, 3051438735)\n",
      "train.csv:8000 (userId, eventId)=(2338481531, 2525447278)\n",
      "train.csv:8500 (userId, eventId)=(2489551967, 520657921)\n",
      "train.csv:9000 (userId, eventId)=(2650493630, 87962584)\n",
      "train.csv:9500 (userId, eventId)=(2791418962, 4223848259)\n",
      "train.csv:10000 (userId, eventId)=(2903662804, 2791462807)\n",
      "train.csv:10500 (userId, eventId)=(3036141956, 3929507420)\n",
      "train.csv:11000 (userId, eventId)=(3176074542, 3459485614)\n",
      "train.csv:11500 (userId, eventId)=(3285425249, 2271782630)\n",
      "train.csv:12000 (userId, eventId)=(3410667855, 1063772489)\n",
      "train.csv:12500 (userId, eventId)=(3531604778, 2584839423)\n",
      "train.csv:13000 (userId, eventId)=(3686871863, 53495098)\n",
      "train.csv:13500 (userId, eventId)=(3833637800, 2415873572)\n",
      "train.csv:14000 (userId, eventId)=(3944021305, 2096772901)\n",
      "train.csv:14500 (userId, eventId)=(4075466480, 3567240505)\n",
      "train.csv:15000 (userId, eventId)=(4197193550, 1628057176)\n",
      "生成预测数据...\n",
      "\n",
      "test.csv:500 (userId, eventId)=(182290053, 2529072432)\n",
      "test.csv:1000 (userId, eventId)=(433510318, 4244463632)\n",
      "test.csv:1500 (userId, eventId)=(632808865, 2845303452)\n",
      "test.csv:2000 (userId, eventId)=(813611885, 2036538169)\n",
      "test.csv:2500 (userId, eventId)=(1010701404, 303459881)\n",
      "test.csv:3000 (userId, eventId)=(1210932037, 2529072432)\n",
      "test.csv:3500 (userId, eventId)=(1452921099, 2705317682)\n",
      "test.csv:4000 (userId, eventId)=(1623287180, 1626678328)\n",
      "test.csv:4500 (userId, eventId)=(1855201342, 2603032829)\n",
      "test.csv:5000 (userId, eventId)=(2083900381, 2529072432)\n",
      "test.csv:5500 (userId, eventId)=(2318415276, 2509151803)\n",
      "test.csv:6000 (userId, eventId)=(2528161539, 4025975316)\n",
      "test.csv:6500 (userId, eventId)=(2749110768, 4244406355)\n",
      "test.csv:7000 (userId, eventId)=(2927772127, 1532377761)\n",
      "test.csv:7500 (userId, eventId)=(3199685636, 1776393554)\n",
      "test.csv:8000 (userId, eventId)=(3393388475, 680270887)\n",
      "test.csv:8500 (userId, eventId)=(3601169721, 154434302)\n",
      "test.csv:9000 (userId, eventId)=(3828963415, 3067222491)\n",
      "test.csv:9500 (userId, eventId)=(4018723397, 2522610844)\n",
      "test.csv:10000 (userId, eventId)=(4180064266, 2658555390)\n"
     ]
    }
   ],
   "source": [
    "RS = RecommonderSystem()\n",
    "print (\"生成训练数据...\\n\")\n",
    "generateRSData(RS,train=True,  header=True)\n",
    "\n",
    "print (\"生成预测数据...\\n\")\n",
    "generateRSData(RS, train=False, header=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "时间、地点等特征都没有处理了，可以考虑用户看到event的时间与event开始时间的差、用户地点和event地点的差异。。。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
